package exp;

import java.io.File;
import java.util.List;

import tools.ObjectTools;
import tools.webkit.Crawler;
import tools.webkit.Webpage;

/**
 * Download the URLs for the first time
 * <br>
 * Since all urls on twitter is shortened. This step is essential.
 * 
 * @author Lanjun
 * 
 */
public class EXP3_1 {
	public void run(String inputPath, String outputPath) {
		for (File file : (new File(inputPath)).listFiles()) {
			if ((new File(outputPath + file.getName())).exists()) {
				System.err.println("EXP3_1: " + file.getAbsolutePath() + " already processed!");
				continue;
			}
			
			@SuppressWarnings("unchecked")
			List<String> urls = (List<String>) ObjectTools.readFromFile(file);			
			Crawler c = new Crawler(10);
			List<Webpage> webpages = c.crawl(urls);
			ObjectTools.writeToFile(webpages, outputPath + file.getName());
		}
	}
	
	public static void main(String[] args) {
		String inputPath = "data/_newData/obj/urlContent1/";
		String outputPath = "data/_newData/obj/urlContent1_1/";
		
		EXP3_1 exp = new EXP3_1();
		exp.run(inputPath, outputPath);
	}
}
